{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import platform\n",
    "print(\"PyTorch version:{}\".format(torch.__version__))\n",
    "print(\"Python version:{}\".format(platform.python_version()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第6章 PyTorch基础"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.1.1 Tensor的数据类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.FloatTensor(2,3)\n",
    "b = torch.FloatTensor([2,3,4,5])\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.IntTensor(2,3)\n",
    "b = torch.IntTensor([2,3,4,5])\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.rand(2,3)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "a = torch.randn(2,3)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "#a = torch.range(1,20,1)\n",
    "a = torch.arange(1,20,1)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.zeros(2,3)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.1.2 Tensor的运算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.abs(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.randn(2,3)\n",
    "print(b)\n",
    "\n",
    "c = torch.add(a,b)\n",
    "print(c)\n",
    "\n",
    "d = torch.randn(2,3)\n",
    "print(d)\n",
    "\n",
    "e = torch.add(d,10)\n",
    "print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.clamp(a, -0.1, 0.1)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.randn(2,3)\n",
    "print(b)\n",
    "\n",
    "c = torch.div(a,b)\n",
    "print(c)\n",
    "\n",
    "d = torch.randn(2,3)\n",
    "print(d)\n",
    "\n",
    "e = torch.div(d,10)\n",
    "print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.randn(2,3)\n",
    "print(b)\n",
    "\n",
    "c = torch.mul(a,b)\n",
    "print(c)\n",
    "\n",
    "d = torch.randn(2,3)\n",
    "print(d)\n",
    "\n",
    "e = torch.mul(d,10)\n",
    "print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.pow(a, 2)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.randn(3, 2)\n",
    "print(b)\n",
    "c = torch.mm(a, b)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.randn(2,3)\n",
    "print(a)\n",
    "\n",
    "b = torch.randn(3)\n",
    "print(b)\n",
    "\n",
    "c = torch.mv(a, b)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.1.3 搭建一个简易神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "batch_n = 100\n",
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "\n",
    "x = torch.randn(batch_n, input_data)\n",
    "y = torch.randn(batch_n, output_data)\n",
    "\n",
    "w1 = torch.randn(input_data, hidden_layer)\n",
    "w2 = torch.randn(hidden_layer, output_data)\n",
    "\n",
    "epoch_n = 20\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for epoch in range(epoch_n):\n",
    "    h1 = x.mm(w1) #100*1000\n",
    "    h1 = h1.clamp(min = 0)\n",
    "    y_pred = h1.mm(w2) #100*10\n",
    "    \n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(\"Epoch:{}, Loss:{:.4f}\".format(epoch,loss))\n",
    "    \n",
    "    grad_y_pred = 2*(y_pred - y)\n",
    "    grad_w2 = h1.t().mm(grad_y_pred)\n",
    "    \n",
    "    grad_h = grad_y_pred.clone()\n",
    "    grad_h = grad_h.mm(w2.t())\n",
    "    grad_h.clamp_(min=0)\n",
    "    \n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "    w1 -= learning_rate*grad_w1\n",
    "w2 -= learning_rate*grad_w2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.2.1 torch.autograd和Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "#from torch.autograd import Variable\n",
    "batch_n = 100\n",
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "\n",
    "#x = Variable(torch.randn(batch_n, input_data), requires_grad = False)\n",
    "#y = Variable(torch.randn(batch_n, output_data), requires_grad = False)\n",
    "x = torch.randn((batch_n, input_data), requires_grad = False)\n",
    "y = torch.randn((batch_n, output_data), requires_grad = False)\n",
    "\n",
    "#w1 = Variable(torch.randn(input_data, hidden_layer), requires_grad = True)\n",
    "#w2 = Variable(torch.randn(hidden_layer, output_data), requires_grad = True)\n",
    "w1 = torch.randn((input_data, hidden_layer), requires_grad = True)\n",
    "w2 = torch.randn((hidden_layer, output_data), requires_grad = True)\n",
    "\n",
    "epoch_n = 20\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for epoch in range(epoch_n):\n",
    "    y_pred = x.mm(w1).clamp(min = 0).mm(w2)\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(\"Epoch:{}, Loss:{:.4f}\".format(epoch,loss.data))\n",
    "    \n",
    "    loss.backward()\n",
    "    \n",
    "    w1.data -= learning_rate*w1.grad.data\n",
    "    w2.data -= learning_rate*w2.grad.data\n",
    "    \n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.2.2 自定义传播函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "#from torch.autograd import Variable\n",
    "batch_n = 64\n",
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "    \n",
    "    def forward(self, input, w1, w2):\n",
    "        x = torch.mm(input, w1)\n",
    "        x = torch.clamp(x, min = 0)\n",
    "        x =torch.mm(x, w2)\n",
    "        return x\n",
    "    \n",
    "    def backward(self):\n",
    "        pass\n",
    "    \n",
    "model = Model()\n",
    "\n",
    "#x = Variable(torch.randn(batch_n, input_data), requires_grad = False)\n",
    "#y = Variable(torch.randn(batch_n, output_data), requires_grad = False)\n",
    "x = torch.randn((batch_n, input_data), requires_grad = False)\n",
    "y = torch.randn((batch_n, output_data), requires_grad = False)\n",
    "\n",
    "#w1 = Variable(torch.randn(input_data, hidden_layer), requires_grad = True)\n",
    "#w2 = Variable(torch.randn(hidden_layer, output_data), requires_grad = True)\n",
    "w1 = torch.randn((input_data, hidden_layer), requires_grad = True)\n",
    "w2 = torch.randn((hidden_layer, output_data), requires_grad = True)\n",
    "\n",
    "epoch_n = 30\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for epoch in range(epoch_n):\n",
    "    y_pred = model(x, w1, w2)\n",
    "    \n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(\"Epoch:{}, Loss:{:.4f}\".format(epoch,loss.data))\n",
    "    loss.backward()\n",
    "    \n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "    \n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.3.1 PyTorch之torch.nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Sequential "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "models = torch.nn.Sequential(\n",
    "torch.nn.Linear(input_data, hidden_layer),\n",
    "torch.nn.ReLU(),\n",
    "torch.nn.Linear(hidden_layer, output_data)\n",
    ")\n",
    "print(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "from collections import OrderedDict\n",
    "models = torch.nn.Sequential(OrderedDict([\n",
    "(\"Line1\",torch.nn.Linear(input_data, hidden_layer)),\n",
    "(\"Relu1\",torch.nn.ReLU()),\n",
    "(\"Line2\",torch.nn.Linear(hidden_layer, output_data))])\n",
    ")\n",
    "print(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "loss_f = torch.nn.MSELoss()\n",
    "x = Variable(torch.randn(100,100))\n",
    "y = Variable(torch.randn(100,100))\n",
    "loss = loss_f(x,y)\n",
    "print(loss.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "loss_f = torch.nn.L1Loss()\n",
    "x = Variable(torch.randn(100,100))\n",
    "y = Variable(torch.randn(100,100))\n",
    "loss = loss_f(x,y)\n",
    "print(loss.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "loss_f = torch.nn.CrossEntropyLoss()\n",
    "x = Variable(torch.randn(3, 5))\n",
    "y = Variable(torch.LongTensor(3).random_(5))\n",
    "loss = loss_f(x,y)\n",
    "print(loss.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "batch_n = 100\n",
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "x = Variable(torch.randn(batch_n, input_data), requires_grad = False)\n",
    "y = Variable(torch.randn(batch_n, output_data), requires_grad = False)\n",
    "\n",
    "models = torch.nn.Sequential(\n",
    "torch.nn.Linear(input_data, hidden_layer),\n",
    "torch.nn.ReLU(),\n",
    "torch.nn.Linear(hidden_layer, output_data)\n",
    ")\n",
    "\n",
    "epoch_n = 10000\n",
    "learning_rate = 1e-4\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "for epoch in range(epoch_n):\n",
    "    y_pred = models(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    if epoch%1000 == 0:\n",
    "        print(\"Epoch:{}, Loss:{:.4f}\".format(epoch,loss.data))\n",
    "    models.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in models.parameters():\n",
    "        param.data -= param.grad.data*learning_rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.3.2 PyTorch之torch.optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "#from torch.autograd import Variable\n",
    "batch_n = 100\n",
    "hidden_layer = 100\n",
    "input_data = 1000\n",
    "output_data = 10\n",
    "\n",
    "#x = Variable(torch.randn(batch_n, input_data), requires_grad = False)\n",
    "#y = Variable(torch.randn(batch_n, output_data), requires_grad=False)\n",
    "x = torch.randn((batch_n, input_data), requires_grad = False)\n",
    "y = torch.randn((batch_n, output_data), requires_grad=False)\n",
    "\n",
    "models = torch.nn.Sequential(\n",
    "torch.nn.Linear(input_data, hidden_layer),\n",
    "torch.nn.ReLU(),\n",
    "torch.nn.Linear(hidden_layer, output_data)\n",
    ")\n",
    "\n",
    "epoch_n = 10000\n",
    "learning_rate = 1e-4\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "optimzer = torch.optim.Adam(models.parameters(), lr = learning_rate)\n",
    "\n",
    "for epoch in range(epoch_n):\n",
    "    y_pred = models(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(\"Epoch:{}, Loss:{:.4f}\".format(epoch,loss.data))\n",
    "    optimzer.zero_grad()\n",
    "    \n",
    "    loss.backward()\n",
    "    optimzer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.4.1 torch和torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform=transforms.Compose([transforms.ToTensor(),\n",
    "                              transforms.Normalize(mean=[0.5],std=[0.5])])\n",
    "\n",
    "data_train = datasets.MNIST(root = \"./data/\",\n",
    "                            transform=transform,\n",
    "                            train = True,\n",
    "                            download = True)\n",
    "\n",
    "data_test = datasets.MNIST(root=\"./data/\",\n",
    "                           transform=transform,\n",
    "                           train = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader_train = torch.utils.data.DataLoader(dataset=data_train,\n",
    "                                                batch_size = 64,\n",
    "                                                shuffle = True,\n",
    "                                               )\n",
    "\n",
    "data_loader_test = torch.utils.data.DataLoader(dataset=data_test,\n",
    "                                               batch_size = 64,\n",
    "                                               shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images, labels = next(iter(data_loader_train))\n",
    "\n",
    "img = torchvision.utils.make_grid(images)\n",
    "img = img.numpy().transpose(1,2,0)\n",
    "\n",
    "std = [0.5]\n",
    "mean = [0.5]\n",
    "img = img*std+mean\n",
    "print([labels[i] for i in range(64)])\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.conv1=torch.nn.Sequential(\n",
    "            torch.nn.Conv2d(1,64,kernel_size=3,stride=1,padding=1),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Conv2d(64,128,kernel_size=3,stride=1,padding=1),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.MaxPool2d(stride=2,kernel_size=2))\n",
    "        self.dense=torch.nn.Sequential(\n",
    "            torch.nn.Linear(14*14*128,1024),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Dropout(p=0.5),\n",
    "            torch.nn.Linear(1024, 10))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = x.view(-1, 14*14*128)\n",
    "        x = self.dense(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model()\n",
    "cost = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    running_loss = 0.0\n",
    "    running_correct = 0\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    print(\"-\"*10)\n",
    "    \n",
    "    for data in data_loader_train:\n",
    "        X_train, y_train = data\n",
    "        X_train, y_train = Variable(X_train), Variable(y_train)\n",
    "        outputs = model(X_train)\n",
    "        _,pred = torch.max(outputs.data, 1)\n",
    "        optimizer.zero_grad()\n",
    "        loss = cost(outputs, y_train)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()     \n",
    "        running_loss += loss.data\n",
    "        running_correct += torch.sum(pred == y_train.data)\n",
    "\n",
    "    testing_correct = 0    \n",
    "    for data in data_loader_test:\n",
    "        X_test, y_test = data\n",
    "        X_test, y_test = Variable(X_test), Variable(y_test)\n",
    "        outputs = model(X_test)\n",
    "        _, pred = torch.max(outputs.data, 1)\n",
    "        testing_correct += torch.sum(pred == y_test.data)\n",
    "    print(\"Loss is:{:.4f}, Train Accuracy is:{:.4f}%, Test Accuracy is:{:.4f}\".format(running_loss/len(data_train),100*running_correct/len(data_train),\n",
    "                 100*testing_correct/len(data_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader_test = torch.utils.data.DataLoader(dataset=data_test,\n",
    "                                               batch_size = 4,\n",
    "                                               shuffle = True)\n",
    "\n",
    "X_test, y_test = next(iter(data_loader_test))\n",
    "inputs = Variable(X_test)\n",
    "pred = model(inputs)\n",
    "_,pred = torch.max(pred, 1)\n",
    "print(\"Predict Label is:\", [ i for i in pred.data])\n",
    "print(\"Real Label is:\",[i for i in y_test])\n",
    "\n",
    "img = torchvision.utils.make_grid(X_test)\n",
    "img = img.numpy().transpose(1,2,0)\n",
    "\n",
    "std = [0.5,0.5,0.5]\n",
    "mean = [0.5,0.5,0.5]\n",
    "img = img*std+mean\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
